



<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      <meta http-equiv="x-ua-compatible" content="ie=edge">
      
      
      
      
        <meta name="lang:clipboard.copy" content="Copy to clipboard">
      
        <meta name="lang:clipboard.copied" content="Copied to clipboard">
      
        <meta name="lang:search.language" content="en">
      
        <meta name="lang:search.pipeline.stopwords" content="True">
      
        <meta name="lang:search.pipeline.trimmer" content="True">
      
        <meta name="lang:search.result.none" content="No matching documents">
      
        <meta name="lang:search.result.one" content="1 matching document">
      
        <meta name="lang:search.result.other" content="# matching documents">
      
        <meta name="lang:search.tokenizer" content="[\s\-]+">
      
      <link rel="shortcut icon" href="../assets/images/favicon.png">
      <meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
    
    
      
        <title>SQuAD - sotabencheval Docs</title>
      
    
    
      <link rel="stylesheet" href="../assets/stylesheets/application.0284f74d.css">
      
        <link rel="stylesheet" href="../assets/stylesheets/application-palette.01803549.css">
      
      
        
        
        <meta name="theme-color" content="#00bcd4">
      
    
    
      <script src="../assets/javascripts/modernizr.74668098.js"></script>
    
    
      
        <link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
        <style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
      
    
    <link rel="stylesheet" href="../assets/fonts/material-icons.css">
    
    
    
      
    
    
  </head>
  
    
    
    <body dir="ltr" data-md-color-primary="cyan" data-md-color-accent="cyan">
  
    <svg class="md-svg">
      <defs>
        
        
      </defs>
    </svg>
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
    
      <a href="#squad" tabindex="1" class="md-skip">
        Skip to content
      </a>
    
    
      <header class="md-header" data-md-component="header">
  <nav class="md-header-nav md-grid">
    <div class="md-flex">
      <div class="md-flex__cell md-flex__cell--shrink">
        <a href=".." title="sotabencheval Docs" class="md-header-nav__button md-logo">
          
            <i class="md-icon"></i>
          
        </a>
      </div>
      <div class="md-flex__cell md-flex__cell--shrink">
        <label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
      </div>
      <div class="md-flex__cell md-flex__cell--stretch">
        <div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
          
            <span class="md-header-nav__topic">
              sotabencheval Docs
            </span>
            <span class="md-header-nav__topic">
              
                SQuAD
              
            </span>
          
        </div>
      </div>
      <div class="md-flex__cell md-flex__cell--shrink">
        
          <label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
          
<div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
      <label class="md-icon md-search__icon" for="__search"></label>
      <button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
        &#xE5CD;
      </button>
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" data-md-scrollfix>
        <div class="md-search-result" data-md-component="result">
          <div class="md-search-result__meta">
            Type to start searching
          </div>
          <ol class="md-search-result__list"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
        
      </div>
      
    </div>
  </nav>
</header>
    
    <div class="md-container">
      
        
      
      
      <main class="md-main">
        <div class="md-main__inner md-grid" data-md-component="container">
          
            
              <div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    <nav class="md-nav md-nav--primary" data-md-level="0">
  <label class="md-nav__title md-nav__title--site" for="__drawer">
    <a href=".." title="sotabencheval Docs" class="md-nav__button md-logo">
      
        <i class="md-icon"></i>
      
    </a>
    sotabencheval Docs
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
      
      


  <li class="md-nav__item">
    <a href=".." title="Welcome to sotabencheval!" class="md-nav__link">
      Welcome to sotabencheval!
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../ade20k/" title="ADE20K" class="md-nav__link">
      ADE20K
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../coco/" title="COCO" class="md-nav__link">
      COCO
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../imagenet/" title="ImageNet" class="md-nav__link">
      ImageNet
    </a>
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../pascalvoc/" title="PASCAL VOC 2012" class="md-nav__link">
      PASCAL VOC 2012
    </a>
  </li>

    
      
      
      

  


  <li class="md-nav__item md-nav__item--active">
    
    <input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
    
      
    
    
      <label class="md-nav__link md-nav__link--active" for="__toc">
        SQuAD
      </label>
    
    <a href="./" title="SQuAD" class="md-nav__link md-nav__link--active">
      SQuAD
    </a>
    
      
<nav class="md-nav md-nav--secondary">
  
  
    
  
  
    <label class="md-nav__title" for="__toc">Table of contents</label>
    <ul class="md-nav__list" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#getting-started" title="Getting Started" class="md-nav__link">
    Getting Started
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#server-data-location" title="Server Data Location" class="md-nav__link">
    Server Data Location
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-initialize-an-evaluator" title="How Do I Initialize an Evaluator?" class="md-nav__link">
    How Do I Initialize an Evaluator?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-evaluate-predictions" title="How Do I Evaluate Predictions?" class="md-nav__link">
    How Do I Evaluate Predictions?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-cache-evaluation" title="How Do I Cache Evaluation?" class="md-nav__link">
    How Do I Cache Evaluation?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#need-more-help" title="Need More Help?" class="md-nav__link">
    Need More Help?
  </a>
  
</li>
      
      
      
      
      
    </ul>
  
</nav>
    
  </li>

    
      
      
      


  <li class="md-nav__item">
    <a href="../wmt/" title="WMT" class="md-nav__link">
      WMT
    </a>
  </li>

    
  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
              <div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    
<nav class="md-nav md-nav--secondary">
  
  
    
  
  
    <label class="md-nav__title" for="__toc">Table of contents</label>
    <ul class="md-nav__list" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#getting-started" title="Getting Started" class="md-nav__link">
    Getting Started
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#server-data-location" title="Server Data Location" class="md-nav__link">
    Server Data Location
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-initialize-an-evaluator" title="How Do I Initialize an Evaluator?" class="md-nav__link">
    How Do I Initialize an Evaluator?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-evaluate-predictions" title="How Do I Evaluate Predictions?" class="md-nav__link">
    How Do I Evaluate Predictions?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#how-do-i-cache-evaluation" title="How Do I Cache Evaluation?" class="md-nav__link">
    How Do I Cache Evaluation?
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#need-more-help" title="Need More Help?" class="md-nav__link">
    Need More Help?
  </a>
  
</li>
      
      
      
      
      
    </ul>
  
</nav>
                  </div>
                </div>
              </div>
            
          
          <div class="md-content">
            <article class="md-content__inner md-typeset">
              
                
                
                <h1 id="squad">SQuAD</h1>
<p><img alt="SQuAD 2.0 Dataset Examples" src="../img/squad20.png" /></p>
<p>You can view the <a href="https://sotabench.com/benchmarks/question-answering-on-squad11-dev">SQuAD 1.1</a> and
<a href="https://sotabench.com/benchmarks/question-answering-on-squad20-dev">SQuAD 2.0</a> leaderboards.</p>
<h2 id="getting-started">Getting Started</h2>
<p>You'll need the following in the root of your repository:</p>
<ul>
<li><code>sotabench.py</code> file - contains benchmarking logic; the server will run this on each commit</li>
<li><code>requirements.txt</code> file - Python dependencies to be installed before running <code>sotabench.py</code></li>
<li><code>sotabench_setup.sh</code> <em>(optional)</em> - any advanced dependencies or setup, e.g. compilation</li>
</ul>
<p>You can write whatever you want in your <code>sotabench.py</code> file to get model predictions on the SQuAD dataset.</p>
<p>But you will need to record your results for the server, and you'll want to avoid doing things like
downloading the dataset on the server. So you should:</p>
<ul>
<li><strong>Point to the server SQuAD data path</strong> - popular datasets are pre-downloaded on the server.</li>
<li><strong>Include an Evaluation object</strong> in <code>sotabench.py</code> file to record the results.</li>
<li><strong>Use Caching</strong> <em>(optional)</em> - to speed up evaluation by hashing the first batch of predictions.</li>
</ul>
<p>We explain how to do these various steps below.</p>
<h2 id="server-data-location">Server Data Location</h2>
<p>The SQuAD development data is located in the root of your repository on the server at <code>.data/nlp/squad</code>.
In this folder is contained:</p>
<ul>
<li><code>dev-v1.1.json</code> - containing SQuAD v1.1 development dataset</li>
<li><code>dev-v2.0.json</code> - containing SQuAD v2.0 development dataset</li>
</ul>
<p>Your local files may have a different file directory structure, so you
can use control flow like below to change the data path if the script is being
run on sotabench servers:</p>
<div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">sotabencheval.utils</span> <span class="kn">import</span> <span class="n">is_server</span>

<span class="k">if</span> <span class="n">is_server</span><span class="p">():</span>
    <span class="n">DATA_ROOT</span> <span class="o">=</span> <span class="s1">&#39;.data/nlp/squad&#39;</span>
<span class="k">else</span><span class="p">:</span> <span class="c1"># local settings</span>
    <span class="n">DATA_ROOT</span> <span class="o">=</span> <span class="s1">&#39;/home/ubuntu/my_data/&#39;</span>
</pre></div>


<p>This will detect if <code>sotabench.py</code> is being run on the server and change behaviour accordingly.</p>
<h2 id="how-do-i-initialize-an-evaluator">How Do I Initialize an Evaluator?</h2>
<p>Add this to your code - before you start batching over the dataset and making predictions:</p>
<div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">sotabencheval.question_answering</span> <span class="kn">import</span> <span class="n">SQuADEvaluator</span><span class="p">,</span> <span class="n">SQuADVersion</span>

<span class="c1"># for SQuAD v1.1</span>
<span class="n">evaluator</span> <span class="o">=</span> <span class="n">SQuADEvaluator</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s1">&#39;My Super Model&#39;</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="n">SQuADVersion</span><span class="o">.</span><span class="n">V11</span><span class="p">)</span>
<span class="c1"># for SQuAD v2.0</span>
<span class="n">evaluator</span> <span class="o">=</span> <span class="n">SQuADEvaluator</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s1">&#39;My Super Model&#39;</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="n">SQuADVersion</span><span class="o">.</span><span class="n">V20</span><span class="p">)</span>
</pre></div>


<p>If you are reproducing a model from a paper, then you can enter the arXiv ID. If you
put in the same model name string as on the
<a href="https://sotabench.com/benchmarks/question-answering-on-squad11-dev">SQuAD 1.1</a> or
<a href="https://sotabench.com/benchmarks/question-answering-on-squad20-dev">SQuAD 2.0</a> leaderboard
then you will enable direct comparison with the paper's model. For example:</p>
<div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">sotabencheval.question_answering</span> <span class="kn">import</span> <span class="n">SQuADEvaluator</span><span class="p">,</span> <span class="n">SQuADVersion</span>

<span class="n">evaluator</span> <span class="o">=</span> <span class="n">SQuADEvaluator</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s1">&#39;SpanBERT&#39;</span><span class="p">,</span>
                           <span class="n">paper_arxiv_id</span><span class="o">=</span><span class="s1">&#39;1907.10529&#39;</span><span class="p">,</span>
                           <span class="n">version</span><span class="o">=</span><span class="n">SQuADVersion</span><span class="o">.</span><span class="n">V20</span><span class="p">)</span>
</pre></div>


<p>The above will directly compare with the result of the paper when run on the server.</p>
<h2 id="how-do-i-evaluate-predictions">How Do I Evaluate Predictions?</h2>
<p>The evaluator object has an <code>.add(answers: Dict[str, str])</code> method to submit predictions by batch or in full.</p>
<p>For SQuAD the expected input is a dictionary, where keys are question ids and values are text answers.
For unanswerable questions the answer should be an empty string. For example:</p>
<div class="codehilite"><pre><span></span><span class="p">{</span><span class="s2">&quot;57296d571d04691400779413&quot;</span><span class="p">:</span> <span class="s2">&quot;itself&quot;</span><span class="p">,</span> <span class="s2">&quot;5a89117e19b91f001a626f2d&quot;</span><span class="p">:</span> <span class="s2">&quot;&quot;</span><span class="p">}</span>
</pre></div>


<p>You can do this all at once in a single call to <code>add()</code>, but more naturally, you will
probably loop over the dataset and call the method for the outputs of each batch.
That would look something like this (for a PyTorch example):</p>
<div class="codehilite"><pre><span></span><span class="o">...</span>

<span class="n">evaluator</span> <span class="o">=</span> <span class="n">SQuADEvaluator</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s1">&#39;My Super Model&#39;</span><span class="p">,</span>
                           <span class="n">paper_arxiv_id</span><span class="o">=</span><span class="s2">&quot;1710.10723&quot;</span><span class="p">,</span>
                           <span class="n">version</span><span class="o">=</span><span class="n">SQuADVersion</span><span class="o">.</span><span class="n">V11</span><span class="p">)</span>

<span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data_loader</span><span class="p">):</span>
        <span class="o">...</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
        <span class="c1"># potentially formatting of the output here to be a dict</span>
        <span class="n">evaluator</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>
</pre></div>


<p>When you are done, you can get the results locally by running:</p>
<div class="codehilite"><pre><span></span><span class="n">evaluator</span><span class="o">.</span><span class="n">get_results</span><span class="p">()</span>
</pre></div>


<p>But for the server you want to save the results by running:</p>
<div class="codehilite"><pre><span></span><span class="n">evaluator</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
</pre></div>


<p>This method serialises the results and model metadata and stores to the server database.</p>
<h2 id="how-do-i-cache-evaluation">How Do I Cache Evaluation?</h2>
<p>Sotabench reruns your script on every commit. This is good because it acts like
continuous integration in checking for bugs and changes, but can be annoying
if the model hasn't changed and evaluation is lengthy.</p>
<p>Fortunately sotabencheval has caching logic that you can use.</p>
<p>The idea is that after the first batch, we hash the model outputs and the
current metrics and this tells us if the model is the same given the dataset.
You can include hashing within an evaluation loop like follows (in the following
example for a PyTorch repository):</p>
<div class="codehilite"><pre><span></span><span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">data_loader</span><span class="p">):</span>
        <span class="o">...</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
        <span class="c1"># potentially formatting of the output here to be a list of dicts</span>
        <span class="n">evaluator</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">evaluator</span><span class="o">.</span><span class="n">cache_exists</span><span class="p">:</span>
            <span class="k">break</span>

<span class="n">evaluator</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
</pre></div>


<p>If the hash is the same as in the server, we infer that the model hasn't changed, so
we simply return hashed results rather than running the whole evaluation again.</p>
<p>Caching is very useful if you have large models, or a repository that is evaluating
multiple models, as it speeds up evaluation significantly.</p>
<h2 id="need-more-help">Need More Help?</h2>
<p>Head on over to the <a href="https://forum.sotabench.com/c/nlp">Natural Language Processing</a> section of the sotabench
forums if you have any questions or difficulties.</p>
                
                  
                
              
              
                


              
            </article>
          </div>
        </div>
      </main>
      
        
<footer class="md-footer">
  
    <div class="md-footer-nav">
      <nav class="md-footer-nav__inner md-grid">
        
          <a href="../pascalvoc/" title="PASCAL VOC 2012" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
            <div class="md-flex__cell md-flex__cell--shrink">
              <i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
            </div>
            <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
              <span class="md-flex__ellipsis">
                <span class="md-footer-nav__direction">
                  Previous
                </span>
                PASCAL VOC 2012
              </span>
            </div>
          </a>
        
        
          <a href="../wmt/" title="WMT" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
            <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
              <span class="md-flex__ellipsis">
                <span class="md-footer-nav__direction">
                  Next
                </span>
                WMT
              </span>
            </div>
            <div class="md-flex__cell md-flex__cell--shrink">
              <i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
            </div>
          </a>
        
      </nav>
    </div>
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-footer-copyright">
        
        powered by
        <a href="https://www.mkdocs.org">MkDocs</a>
        and
        <a href="https://squidfunk.github.io/mkdocs-material/">
          Material for MkDocs</a>
      </div>
      
    </div>
  </div>
</footer>
      
    </div>
    
      <script src="../assets/javascripts/application.245445c6.js"></script>
      
      <script>app.initialize({version:"1.0.4",url:{base:".."}})</script>
      
    
  </body>
</html>